package 公司_问答;

import Utils.HttpConnect;

import com.mongodb.BasicDBObject;
import com.mongodb.MongoClient;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.model.UpdateOptions;
import org.apache.http.ProtocolException;
import org.jcp.xml.dsig.internal.dom.Utils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.seimicrawler.xpath.JXDocument;

import javax.rmi.CORBA.Util;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class 公司信息 {
    /**
     * url: http://sns.sseinfo.com/list/company.do
     * 抓取所有 ‘上市公司’ 信息
     */

    static final String COOKIE = "";
    static final int START_PAGE_INDEX = 1;
    static final int END_PAGE_INDEX = 62;
    static final String POST_URL = "http://sns.sseinfo.com/allcompany.do";
    static final int REQUEST_DELAY = 1;
    static Map<String, String> header_map = new HashMap<>();
    static Map<String, String> data_map = new HashMap<>();
    static MongoCollection collection = new MongoClient("192.168.1.173", 27017).getDatabase("IRMB_上证e").getCollection("CompanyInfo");

    static {
        /**
         * 初始化request.header   request.data
         */
        header_map.put("Cookie", COOKIE);
        header_map.put("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");

        data_map.put("code", "0");
        data_map.put("order", "0");
        data_map.put("areaId", "0");
        data_map.put("page", "1");
    }

    public static void main(String[] args) {
        for (int i = START_PAGE_INDEX ; i <= END_PAGE_INDEX ; i++) {
            data_map.put("page", String.valueOf(i));

            List<org.bson.Document> docList = parseCompanyInfo(HttpConnect.HttpPost_OkHttp(POST_URL, header_map, data_map));

            /**
             * 写入库
             */
            docList.forEach(doc -> {
                BasicDBObject obj = new BasicDBObject("$set", doc);
                BasicDBObject query = new BasicDBObject().append("_id", doc.get("_id"));
                collection.updateOne(query, obj, (new UpdateOptions()).upsert(true));
                System.out.println(doc.get("_id") + " 写入数据库");
            });
        }
    }

    public static List<org.bson.Document> parseCompanyInfo(String responseStr){
        /**
         * 解析html
         * 提取数据
         */
        responseStr = responseStr.substring(12, responseStr.length()-2).replaceAll("<\\\\/a>", "</a>").replaceAll("<\\\\/div>", "</div>").replaceAll("\\\\t", "");
        Document doc = Jsoup.parse(responseStr);

        Elements tagDivElements = doc.getElementsByClass("companyBox");
        int lenDivTag = tagDivElements.size();

        List<org.bson.Document> docList = new ArrayList<>();

        for (int i = 0; i < lenDivTag; i++) {
            Element tagDiv = tagDivElements.get(i);
            String[] companyInfo = tagDiv.text().split(" ");
            String url = tagDiv.select("a").get(0).attr("href");

            docList.add(new org.bson.Document("_id", companyInfo[0] + "_" + companyInfo[1])
                                      .append("companyId", companyInfo[0])
                                      .append("companyName", companyInfo[1])
                                      .append("url", url));
        }

        return docList;
    }
}
